{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = '/home/husein/t5/prepare/mesolitica-tpu.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage\n",
    "client = storage.Client()\n",
    "bucket = client.bucket('mesolitica-tpu-general')\n",
    "\n",
    "directory = 't2t-paraphrase-base'\n",
    "model = '50000'\n",
    "\n",
    "!mkdir {directory}\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.data-00000-of-00001')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.data-00000-of-00001')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.index')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.index')\n",
    "\n",
    "blob = bucket.blob(f'{directory}/model.ckpt-{model}.meta')\n",
    "blob.download_to_filename(f'{directory}/model.ckpt-{model}.meta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '3'\n",
    "\n",
    "from tensor2tensor.data_generators import problem\n",
    "from tensor2tensor.data_generators import text_problems\n",
    "from tensor2tensor.data_generators import translate\n",
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor import problems\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger()\n",
    "tf.logging.set_verbosity(tf.logging.DEBUG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "vocab = 'sp10m.cased.t5.model'\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(vocab)\n",
    "\n",
    "class Encoder:\n",
    "    def __init__(self, sp):\n",
    "        self.sp = sp\n",
    "        self.vocab_size = sp.GetPieceSize() + 100\n",
    "    \n",
    "    def encode(self, s):\n",
    "        return self.sp.EncodeAsIds(s)\n",
    "    \n",
    "    def decode(self, ids, strip_extraneous=False):\n",
    "        return self.sp.DecodeIds(list(ids))\n",
    "    \n",
    "encoder = Encoder(sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "from glob import glob\n",
    "\n",
    "@registry.register_problem\n",
    "class Seq2Seq(text_problems.Text2TextProblem):\n",
    "\n",
    "    @property\n",
    "    def approx_vocab_size(self):\n",
    "        return 32100\n",
    "    \n",
    "    @property\n",
    "    def is_generate_per_split(self):\n",
    "        return False\n",
    "            \n",
    "    def feature_encoders(self, data_dir):\n",
    "        encoder = Encoder(sp)\n",
    "        return {\n",
    "            \"inputs\": encoder,\n",
    "            \"targets\": encoder\n",
    "        }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = os.path.expanduser('t2t-paraphrase/data')\n",
    "TMP_DIR = os.path.expanduser('t2t-paraphrase/tmp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "PROBLEM = 'seq2_seq'\n",
    "t2t_problem = problems.problem(PROBLEM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "ckpt_path = tf.train.latest_checkpoint('t2t-paraphrase-base')\n",
    "ckpt_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_gan/python/estimator/tpu_gan_estimator.py:42: The name tf.estimator.tpu.TPUEstimator is deprecated. Please use tf.compat.v1.estimator.tpu.TPUEstimator instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from tensor2tensor import models\n",
    "from tensor2tensor import problems\n",
    "from tensor2tensor.layers import common_layers\n",
    "from tensor2tensor.utils import trainer_lib\n",
    "from tensor2tensor.utils import t2t_model\n",
    "from tensor2tensor.utils import registry\n",
    "from tensor2tensor.utils import metrics\n",
    "from tensor2tensor.data_generators import problem\n",
    "from tensor2tensor.data_generators import text_problems\n",
    "from tensor2tensor.data_generators import translate\n",
    "from tensor2tensor.utils import registry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensor2tensor.layers import modalities\n",
    "from tensor2tensor.layers import common_layers\n",
    "\n",
    "def top_p_logits(logits, p):\n",
    "    with tf.variable_scope('top_p_logits'):\n",
    "        logits_sort = tf.sort(logits, direction = 'DESCENDING')\n",
    "        probs_sort = tf.nn.softmax(logits_sort)\n",
    "        probs_sums = tf.cumsum(probs_sort, axis = 1, exclusive = True)\n",
    "        logits_masked = tf.where(\n",
    "            probs_sums < p, logits_sort, tf.ones_like(logits_sort) * 1000\n",
    "        )  # [batchsize, vocab]\n",
    "        min_logits = tf.reduce_min(\n",
    "            logits_masked, axis = 1, keepdims = True\n",
    "        )  # [batchsize, 1]\n",
    "        return tf.where(\n",
    "            logits < min_logits,\n",
    "            tf.ones_like(logits, dtype = logits.dtype) * -1e10,\n",
    "            logits,\n",
    "        )\n",
    "\n",
    "\n",
    "def sample(translate_model, features):\n",
    "    logits, losses = translate_model(features)\n",
    "    logits_shape = common_layers.shape_list(logits)\n",
    "    logits_p = logits[:,0,:,0,:] / translate_model.hparams.sampling_temp\n",
    "    logits_p = top_p_logits(logits_p, translate_model.hparams.top_p)\n",
    "    reshaped_logits = tf.reshape(logits_p, [-1, logits_shape[-1]])\n",
    "    choices = tf.multinomial(reshaped_logits, 1)\n",
    "    samples = tf.reshape(choices, logits_shape[:-1])\n",
    "    return samples, logits, losses\n",
    "\n",
    "def nucleus_sampling(translate_model, features, decode_length):\n",
    "    \"\"\"A slow greedy inference method.\n",
    "    Quadratic time in decode_length.\n",
    "    Args:\n",
    "      features: an map of string to `Tensor`\n",
    "      decode_length: an integer.  How many additional timesteps to decode.\n",
    "    Returns:\n",
    "      A dict of decoding results {\n",
    "          \"outputs\": integer `Tensor` of decoded ids of shape\n",
    "              [batch_size, <= decode_length] if beam_size == 1 or\n",
    "              [batch_size, top_beams, <= decode_length]\n",
    "          \"scores\": None\n",
    "          \"logits\": `Tensor` of shape [batch_size, time, 1, 1, vocab_size].\n",
    "          \"losses\": a dictionary: {loss-name (string): floating point `Scalar`}\n",
    "      }\n",
    "    \"\"\"\n",
    "    if not features:\n",
    "        features = {}\n",
    "    inputs_old = None\n",
    "    if 'inputs' in features and len(features['inputs'].shape) < 4:\n",
    "        inputs_old = features['inputs']\n",
    "        features['inputs'] = tf.expand_dims(features['inputs'], 2)\n",
    "    # Save the targets in a var and reassign it after the tf.while loop to avoid\n",
    "    # having targets being in a 'while' frame. This ensures targets when used\n",
    "    # in metric functions stays in the same frame as other vars.\n",
    "    targets_old = features.get('targets', None)\n",
    "\n",
    "    target_modality = translate_model._problem_hparams.modality['targets']\n",
    "\n",
    "    def infer_step(recent_output, recent_logits, unused_loss):\n",
    "        \"\"\"Inference step.\"\"\"\n",
    "        if not tf.executing_eagerly():\n",
    "            if translate_model._target_modality_is_real:\n",
    "                dim = translate_model._problem_hparams.vocab_size['targets']\n",
    "                if dim is not None and hasattr(\n",
    "                    translate_model._hparams, 'vocab_divisor'\n",
    "                ):\n",
    "                    dim += (-dim) % translate_model._hparams.vocab_divisor\n",
    "                recent_output.set_shape([None, None, None, dim])\n",
    "            else:\n",
    "                recent_output.set_shape([None, None, None, 1])\n",
    "        padded = tf.pad(recent_output, [[0, 0], [0, 1], [0, 0], [0, 0]])\n",
    "        features['targets'] = padded\n",
    "        # This is inefficient in that it generates samples at all timesteps,\n",
    "        # not just the last one, except if target_modality is pointwise.\n",
    "        samples, logits, losses = sample(translate_model, features)\n",
    "        # Concatenate the already-generated recent_output with last timestep\n",
    "        # of the newly-generated samples.\n",
    "        top = translate_model._hparams.top.get(\n",
    "            'targets', modalities.get_top(target_modality)\n",
    "        )\n",
    "        if getattr(top, 'pointwise', False):\n",
    "            cur_sample = samples[:, -1, :, :]\n",
    "        else:\n",
    "            cur_sample = samples[\n",
    "                :, common_layers.shape_list(recent_output)[1], :, :\n",
    "            ]\n",
    "        if translate_model._target_modality_is_real:\n",
    "            cur_sample = tf.expand_dims(cur_sample, axis = 1)\n",
    "            samples = tf.concat([recent_output, cur_sample], axis = 1)\n",
    "        else:\n",
    "            cur_sample = tf.to_int64(tf.expand_dims(cur_sample, axis = 1))\n",
    "            samples = tf.concat([recent_output, cur_sample], axis = 1)\n",
    "            if not tf.executing_eagerly():\n",
    "                samples.set_shape([None, None, None, 1])\n",
    "\n",
    "        # Assuming we have one shard for logits.\n",
    "        logits = tf.concat([recent_logits, logits[:, -1:]], 1)\n",
    "        loss = sum([l for l in losses.values() if l is not None])\n",
    "        return samples, logits, loss\n",
    "\n",
    "    # Create an initial output tensor. This will be passed\n",
    "    # to the infer_step, which adds one timestep at every iteration.\n",
    "    if 'partial_targets' in features:\n",
    "        initial_output = tf.to_int64(features['partial_targets'])\n",
    "        while len(initial_output.get_shape().as_list()) < 4:\n",
    "            initial_output = tf.expand_dims(initial_output, 2)\n",
    "        batch_size = common_layers.shape_list(initial_output)[0]\n",
    "    else:\n",
    "        batch_size = common_layers.shape_list(features['inputs'])[0]\n",
    "        if translate_model._target_modality_is_real:\n",
    "            dim = translate_model._problem_hparams.vocab_size['targets']\n",
    "            if dim is not None and hasattr(\n",
    "                translate_model._hparams, 'vocab_divisor'\n",
    "            ):\n",
    "                dim += (-dim) % translate_model._hparams.vocab_divisor\n",
    "            initial_output = tf.zeros(\n",
    "                (batch_size, 0, 1, dim), dtype = tf.float32\n",
    "            )\n",
    "        else:\n",
    "            initial_output = tf.zeros((batch_size, 0, 1, 1), dtype = tf.int64)\n",
    "    # Hack: foldl complains when the output shape is less specified than the\n",
    "    # input shape, so we confuse it about the input shape.\n",
    "    initial_output = tf.slice(\n",
    "        initial_output, [0, 0, 0, 0], common_layers.shape_list(initial_output)\n",
    "    )\n",
    "    target_modality = translate_model._problem_hparams.modality['targets']\n",
    "    if (\n",
    "        target_modality == modalities.ModalityType.CLASS_LABEL\n",
    "        or translate_model._problem_hparams.get('regression_targets')\n",
    "    ):\n",
    "        decode_length = 1\n",
    "    else:\n",
    "        if 'partial_targets' in features:\n",
    "            prefix_length = common_layers.shape_list(\n",
    "                features['partial_targets']\n",
    "            )[1]\n",
    "        else:\n",
    "            prefix_length = common_layers.shape_list(features['inputs'])[1]\n",
    "        decode_length = prefix_length + decode_length\n",
    "\n",
    "    # Initial values of result, logits and loss.\n",
    "    result = initial_output\n",
    "    vocab_size = translate_model._problem_hparams.vocab_size['targets']\n",
    "    if vocab_size is not None and hasattr(\n",
    "        translate_model._hparams, 'vocab_divisor'\n",
    "    ):\n",
    "        vocab_size += (-vocab_size) % translate_model._hparams.vocab_divisor\n",
    "    if translate_model._target_modality_is_real:\n",
    "        logits = tf.zeros((batch_size, 0, 1, vocab_size))\n",
    "        logits_shape_inv = [None, None, None, None]\n",
    "    else:\n",
    "        # tensor of shape [batch_size, time, 1, 1, vocab_size]\n",
    "        logits = tf.zeros((batch_size, 0, 1, 1, vocab_size))\n",
    "        logits_shape_inv = [None, None, None, None, None]\n",
    "    if not tf.executing_eagerly():\n",
    "        logits.set_shape(logits_shape_inv)\n",
    "\n",
    "    loss = 0.0\n",
    "\n",
    "    def while_exit_cond(\n",
    "        result, logits, loss\n",
    "    ):  # pylint: disable=unused-argument\n",
    "        \"\"\"Exit the loop either if reach decode_length or EOS.\"\"\"\n",
    "        length = common_layers.shape_list(result)[1]\n",
    "\n",
    "        not_overflow = length < decode_length\n",
    "\n",
    "        if translate_model._problem_hparams.stop_at_eos:\n",
    "\n",
    "            def fn_not_eos():\n",
    "                return tf.not_equal(  # Check if the last predicted element is a EOS\n",
    "                    tf.squeeze(result[:, -1, :, :]), 1\n",
    "                )\n",
    "\n",
    "            not_eos = tf.cond(\n",
    "                # We only check for early stopping if there is at least 1 element (\n",
    "                # otherwise not_eos will crash).\n",
    "                tf.not_equal(length, 0),\n",
    "                fn_not_eos,\n",
    "                lambda: True,\n",
    "            )\n",
    "\n",
    "            return tf.cond(\n",
    "                tf.equal(batch_size, 1),\n",
    "                # If batch_size == 1, we check EOS for early stopping.\n",
    "                lambda: tf.logical_and(not_overflow, not_eos),\n",
    "                # Else, just wait for max length\n",
    "                lambda: not_overflow,\n",
    "            )\n",
    "        return not_overflow\n",
    "\n",
    "    result, logits, loss = tf.while_loop(\n",
    "        while_exit_cond,\n",
    "        infer_step,\n",
    "        [result, logits, loss],\n",
    "        shape_invariants = [\n",
    "            tf.TensorShape([None, None, None, None]),\n",
    "            tf.TensorShape(logits_shape_inv),\n",
    "            tf.TensorShape([]),\n",
    "        ],\n",
    "        back_prop = False,\n",
    "        parallel_iterations = 1,\n",
    "    )\n",
    "    if inputs_old is not None:  # Restore to not confuse Estimator.\n",
    "        features['inputs'] = inputs_old\n",
    "    # Reassign targets back to the previous value.\n",
    "    if targets_old is not None:\n",
    "        features['targets'] = targets_old\n",
    "    losses = {'training': loss}\n",
    "    if 'partial_targets' in features:\n",
    "        partial_target_length = common_layers.shape_list(\n",
    "            features['partial_targets']\n",
    "        )[1]\n",
    "        result = tf.slice(\n",
    "            result, [0, partial_target_length, 0, 0], [-1, -1, -1, -1]\n",
    "        )\n",
    "    return {\n",
    "        'outputs': result,\n",
    "        'scores': None,\n",
    "        'logits': logits,\n",
    "        'losses': losses,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'infer'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting T2TModel mode to 'infer'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.label_smoothing to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.label_smoothing to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.layer_prepostprocess_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.layer_prepostprocess_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.symbol_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.symbol_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.attention_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.attention_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.relu_dropout to 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Setting hparams.relu_dropout to 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function framework at 0x7fa7edcabae8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function framework at 0x7fa7edcabae8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function framework at 0x7fa7edcabae8> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: LIVE_VARS_IN\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:2253: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:2253: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:1374: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/t2t_model.py:1374: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:95: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7fa7ec4ffd08> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function attention_bias_to_padding at 0x7fa7ec4ffd08> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function attention_bias_to_padding at 0x7fa7ec4ffd08> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: \n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/expert_utils.py:621: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/utils/expert_utils.py:621: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function layers at 0x7fa7ec962840> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7fa79c34d470>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Entity <function layers at 0x7fa7ec962840> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7fa79c34d470>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING: Entity <function layers at 0x7fa7ec962840> could not be transformed and will be executed as-is. Please report this to the AutoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: If not matching a CFG node, must be a block statement: <gast.gast.ImportFrom object at 0x7fa79c34d470>\n",
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:1258: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensor2tensor-1.15.7-py3.6.egg/tensor2tensor/models/transformer.py:1258: to_int64 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using variable initializer: uniform_unit_scaling\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'inputs' with symbol_modality_32128_768.bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming feature 'targets' with symbol_modality_32128_768.targets_bottom\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Building model body\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Transforming body output with symbol_modality_32128_768.top\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-11-dfd530a8b54f>:28: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.random.categorical` instead.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-11-dfd530a8b54f>:28: multinomial (from tensorflow.python.ops.random_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.random.categorical` instead.\n"
     ]
    }
   ],
   "source": [
    "class Model:\n",
    "    def __init__(self, HPARAMS = \"transformer_base\", DATA_DIR = 't2t/data'):\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.top_p = tf.placeholder(tf.float32, None)\n",
    "        \n",
    "        self.X_seq_len = tf.count_nonzero(self.X, 1, dtype=tf.int32)\n",
    "        self.maxlen_decode = tf.reduce_max(self.X_seq_len)\n",
    "        #self.maxlen_decode = tf.placeholder(tf.int32, None)\n",
    "        \n",
    "        x = tf.expand_dims(tf.expand_dims(self.X, -1), -1)\n",
    "        y = tf.expand_dims(tf.expand_dims(self.Y, -1), -1)\n",
    "        \n",
    "        features = {\n",
    "            \"inputs\": x,\n",
    "            \"targets\": y,\n",
    "            \"target_space_id\": tf.constant(1, dtype=tf.int32),\n",
    "        }\n",
    "        self.features = features\n",
    "        \n",
    "        Modes = tf.estimator.ModeKeys\n",
    "        hparams = trainer_lib.create_hparams(HPARAMS, data_dir=DATA_DIR, problem_name=PROBLEM)\n",
    "        hparams.filter_size = 3072\n",
    "        hparams.hidden_size = 768\n",
    "        hparams.num_heads = 12\n",
    "        hparams.num_hidden_layers = 8\n",
    "        hparams.vocab_divisor = 128\n",
    "        hparams.dropout = 0.1\n",
    "        \n",
    "        hparams.label_smoothing = 0.0\n",
    "        hparams.shared_embedding_and_softmax_weights = False\n",
    "        hparams.eval_drop_long_sequences = True\n",
    "        hparams.max_length = 1024\n",
    "        hparams.multiproblem_mixing_schedule = 'pretrain'\n",
    "        \n",
    "        hparams.symbol_modality_num_shards = 1\n",
    "        hparams.attention_dropout_broadcast_dims = '0,1'\n",
    "        hparams.relu_dropout_broadcast_dims = '1'\n",
    "        hparams.layer_prepostprocess_dropout_broadcast_dims = '1'\n",
    "        \n",
    "        translate_model = registry.model('transformer')(hparams, Modes.PREDICT)\n",
    "        self.translate_model = translate_model\n",
    "        logits, _ = translate_model(features)\n",
    "        self.logits = logits\n",
    "        translate_model.hparams.top_p = self.top_p\n",
    "        \n",
    "        with tf.variable_scope(tf.get_variable_scope(), reuse=True):\n",
    "            self.fast_result = translate_model._greedy_infer(features, self.maxlen_decode)[\"outputs\"]\n",
    "            self.beam_result = translate_model._beam_decode_slow(\n",
    "                features, self.maxlen_decode, beam_size=3, \n",
    "                top_beams=1, alpha=0.5)[\"outputs\"]\n",
    "            self.nucleus_result = nucleus_sampling(translate_model, features, self.maxlen_decode)[\"outputs\"]\n",
    "            self.nucleus_result = self.nucleus_result[:,:,0,0]\n",
    "        \n",
    "        self.fast_result = tf.identity(self.fast_result, name = 'greedy')\n",
    "        self.beam_result = tf.identity(self.beam_result, name = 'beam')\n",
    "        self.nucleus_result = tf.identity(self.nucleus_result, name = 'nucleus')\n",
    "        \n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from t2t-paraphrase-base/model.ckpt-50000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from t2t-paraphrase-base/model.ckpt-50000\n"
     ]
    }
   ],
   "source": [
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, 't2t-paraphrase-base/model.ckpt-50000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    return re.sub(r'[ ]+', ' ', unidecode(string.replace('\\n', ' '))).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "from malaya.text.rules import normalized_chars\n",
    "\n",
    "def filter_news(string):\n",
    "    string = string.lower()\n",
    "    return 'javascript is disabled' in string or 'requires javascript' in string or 'javascript' in string \\\n",
    "    or 'président' in string\n",
    "\n",
    "def make_cleaning(s, c_dict):\n",
    "    s = s.translate(c_dict)\n",
    "    return s\n",
    "\n",
    "def transformer_textcleaning(string):\n",
    "    \"\"\"\n",
    "    use by any transformer model before tokenization\n",
    "    \"\"\"\n",
    "    string = unidecode(string)\n",
    "    string = ' '.join(\n",
    "        [make_cleaning(w, normalized_chars) for w in string.split()]\n",
    "    )\n",
    "    string = re.sub('\\(dot\\)', '.', string)\n",
    "    string = (\n",
    "        re.sub(re.findall(r'\\<a(.*?)\\>', string)[0], '', string)\n",
    "        if (len(re.findall(r'\\<a (.*?)\\>', string)) > 0)\n",
    "        and ('href' in re.findall(r'\\<a (.*?)\\>', string)[0])\n",
    "        else string\n",
    "    )\n",
    "    string = re.sub(\n",
    "        r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', ' ', string\n",
    "    )\n",
    "    string = string.replace('\\n', ' ')\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip().split()\n",
    "    string = [w for w in string if w[0] != '@']\n",
    "    return ' '.join(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['before', 'test_before', 'after', 'test_after'])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/testset/paraphrase-set.json\n",
    "\n",
    "with open('../summary/paraphrase-set.json') as fopen:\n",
    "    test = json.load(fopen)\n",
    "test.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [01:37<00:00,  1.94s/it]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "batch_size = 20\n",
    "results = []\n",
    "for i in tqdm(range(0, len(test['test_before']), batch_size)):\n",
    "    batch_x = test['test_before'][i: i + batch_size]\n",
    "    batches = []\n",
    "    for b in batch_x:\n",
    "        encoded = encoder.encode(f'parafrasa: {cleaning(b)}')\n",
    "        encoded = encoded[:1023] + [1] \n",
    "        batches.append(encoded)\n",
    "    batches = pad_sequences(batches, padding='post')\n",
    "    g = sess.run(model.fast_result, feed_dict = {model.X:batches})\n",
    "    for b in g:\n",
    "        results.append(encoder.decode(b.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.59612304"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensor2tensor.utils import bleu_hook\n",
    "\n",
    "bleu_hook.compute_bleu(reference_corpus = test['test_after'], \n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Ayahnya pernah menjadi pandai besi dan penemu dan pernah bekerja dengan tali besi di California.',\n",
       " 'Ayahnya pernah menjadi pandai besi dan penemu dan pernah bekerja dengan tali besi di California.',\n",
       " 'Ayahnya pernah menjadi pandai besi dan penemu dan pernah bekerja dengan tali besi di California.')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded = encoder.encode(f\"parafrasa: {cleaning(test['test_before'][0])}\") + [1]\n",
    "f, b, n = sess.run([model.fast_result, model.beam_result, model.nucleus_result], \n",
    "             feed_dict = {model.X: [encoded], model.top_p: 0.7})\n",
    "\n",
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Ayahnya pernah menjadi pandai besi dan penemu dan pernah bekerja di California dengan tali besi.'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['test_after'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('Ayahnya kembali ke Bombay sebagai pemain biola sekolah Rusia yang sudah selesai.',\n",
       " 'Ayahnya kembali ke Bombay sebagai pemain biola sekolah Rusia yang sudah selesai.',\n",
       " 'Ayahnya kembali ke Bombay sebagai pemain biola sekolah Rusia yang sudah selesai.')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded = encoder.encode(f\"parafrasa: {cleaning(test['test_before'][1])}\") + [1]\n",
    "f, b, n = sess.run([model.fast_result, model.beam_result, model.nucleus_result], \n",
    "             feed_dict = {model.X: [encoded], model.top_p: 0.7})\n",
    "\n",
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Bapanya kembali sebagai pemain biola dari Sekolah Rusia ke Bombay.'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['test_after'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('EP bernama Dido Live, dengan tiga daripada tujuh belas trek langsung dalam DVD, dirilis pada 21 Jun 2005 secara eksklusif melalui iTunes Store.',\n",
       " 'Sebuah EP bernama Dido Live, dengan tiga daripada tujuh belas trek langsung dalam DVD, dirilis pada 21 Jun 2005 secara eksklusif melalui iTunes Store.',\n",
       " 'Sebuah EP bernama Dido Live, dengan tiga daripada tujuh belas trek dalam DVD, diterbitkan pada 21 Jun 2005 secara eksklusif secara digital di iTunes Store.')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoded = encoder.encode(f\"parafrasa: {cleaning(test['test_before'][2])}\") + [1]\n",
    "f, b, n = sess.run([model.fast_result, model.beam_result, model.nucleus_result], \n",
    "             feed_dict = {model.X: [encoded], model.top_p: 0.7})\n",
    "\n",
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Sebuah EP yang dinamakan Dido Live dengan tiga daripada tujuh belas lagu langsung dalam DVD dilancarkan secara digital pada 21 Jun 2005 secara eksklusif di iTunes Store.'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['test_after'][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'transformer-base/model.ckpt'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'transformer-base/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'transformer/body/target_space_embedding/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_0/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_1/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_2/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_3/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_4/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_5/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_6/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_6/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_6/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_6/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_7/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/encoder/layer_7/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_7/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_7/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/encoder/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_0/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_1/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_2/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_3/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_4/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_5/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_6/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_6/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_6/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_6/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_6_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/self_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/self_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_7/self_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/self_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/self_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/self_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/encdec_attention/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_7/encdec_attention/multihead_attention/q/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/encdec_attention/multihead_attention/k/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/encdec_attention/multihead_attention/v/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/encdec_attention/multihead_attention/output_transform/kernel/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/ffn/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/ffn/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/body/decoder/layer_7/ffn/conv1/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/ffn/conv1/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/ffn/conv2/kernel/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_7/ffn/conv2/bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_7_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_prepostprocess/layer_norm/layer_norm_scale/Read/ReadVariableOp',\n",
       " 'transformer/body/decoder/layer_prepostprocess/layer_norm/layer_norm_bias/Read/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_4/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_6/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_7/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/body/parallel_0/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_6/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_6/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_7/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/body/decoder/layer_7/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_6_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_7_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer_1/while/body/parallel_0/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_2/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_3/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_4/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_5/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_6/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_7/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/encoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_0_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_1_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_2_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_3_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_4_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_5_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_6_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/encdec_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/encdec_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/encdec_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/encdec_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7/encdec_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7_1/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7_1/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7_1/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7_1/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_7_1/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_10/transformer/transformer/body/decoder/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/target_space_embedding/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/v/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/self_attention/multihead_attention/output_transform/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/conv1/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/conv1/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/conv2/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_0/ffn/conv2/BiasAdd/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/layer_prepostprocess/layer_norm/ReadVariableOp_1',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/q/Tensordot/ReadVariableOp',\n",
       " 'transformer/parallel_0_16/transformer/transformer/body/encoder/layer_1/self_attention/multihead_attention/k/Tensordot/ReadVariableOp',\n",
       " ...]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'greedy' in n.name\n",
    "        or 'beam' in n.name\n",
    "        or 'nucleus' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'modality' not in n.name\n",
    "        and 'Assign' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from transformer-base/model.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from transformer-base/model.ckpt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-29-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-29-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 248 variables.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 248 variables.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 248 variables to const ops.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 248 variables to const ops.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20637 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('transformer-base', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('transformer-base/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "greedy = g.get_tensor_by_name('import/greedy:0')\n",
    "beam = g.get_tensor_by_name('import/beam:0')\n",
    "nucleus = g.get_tensor_by_name('import/nucleus:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_p = g.get_tensor_by_name('import/Placeholder_2:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "g, b, n = test_sess.run([greedy, beam, nucleus], feed_dict = {x:[encoded],\n",
    "                                                             top_p: 0.7})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('EP bernama Dido Live, dengan tiga daripada tujuh belas trek langsung dalam DVD, dirilis pada 21 Jun 2005 secara eksklusif melalui iTunes Store.',\n",
       " 'Sebuah EP bernama Dido Live, dengan tiga daripada tujuh belas trek langsung dalam DVD, dirilis pada 21 Jun 2005 secara eksklusif melalui iTunes Store.',\n",
       " 'Sebuah EP bernama Dido Live, dengan tiga daripada tujuh belas trek langsung dalam DVD, dikeluarkan pada 21 Jun 2005 secara eksklusif melalui iTunes Store.')"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(encoder.decode(f[0].tolist()), \n",
    "encoder.decode(b[0].tolist()), \n",
    "encoder.decode(n[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
